package com.catmiao.spark.rdd.dep

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

/**
 * @title: Spark01_RDD_Dep
 * @projectName spark_study
 * @description: TODO
 * @author ChengMiao
 * @date 2024/2/28 23:20
 */
object Spark02_RDD_Dep {


  def main(args: Array[String]): Unit = {
    val sparkCon = new SparkConf().setMaster("local[*]").setAppName("rdd")

    val sparkContext = new SparkContext(sparkCon)


    val rdd1: RDD[String] = sparkContext.textFile("datas/1.txt")

    // 打印血缘关系
    println(rdd1.dependencies)
    println("********************************")


    val rdd2: RDD[String] = rdd1.flatMap(_.split(" "))
    println(rdd2.dependencies)
    println("********************************")

    val rdd3: RDD[(String, Int)] = rdd2.map(word => (word, 1))
    println(rdd3.dependencies)
    println("********************************")

    val rdd4: RDD[(String, Int)] = rdd3.reduceByKey(_ + _)
    println(rdd4 .dependencies)

    println("********************************")
    rdd4.collect().foreach(println)

    sparkContext.stop()
  }
}
